package com.gt.wc

import com.gt.SCUtil
import org.apache.log4j.Logger
import org.apache.spark.rdd.RDD

object WorldCount_05 {
  private val logger: Logger = Logger.getLogger("WorldCount_02")

  def main(args: Array[String]): Unit = {
    val sc = SCUtil.createLocalSc()

    val rdd1: RDD[String] = sc.textFile("data/a.txt")
    //println(rdd1.toDebugString)
    println(rdd1.dependencies)
    println("********************")

    val rdd2: RDD[String] = rdd1.flatMap(line => line.split(" "))
    //println(rdd2.toDebugString)
    println(rdd2.dependencies)
    println("********************")


    val rdd3: RDD[(String, Int)] = rdd2.map(word => (word, 1))
    //println(rdd3.toDebugString)
    println(rdd3.dependencies)
    println("********************")


    val rdd4: RDD[(String, Int)] = rdd3.reduceByKey(_ + _)
    println(rdd4.toDebugString)
    println(rdd4.dependencies)
    println("********************")


    rdd4.collect().foreach(println)

    sc.stop()
  }

}
